{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pendulum_train [action_gap].ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "last_runtime": {
        "build_target": "",
        "kind": "local"
      }
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bZCvVbB1gbGH",
        "colab_type": "text"
      },
      "source": [
        "***Copyright 2020 Google LLC.***\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_Kwbd8GgW6k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#@title Default title text\n",
        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EezlEB6bGVH-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow.compat.v2 as tf\n",
        "import numpy as np\n",
        "import imp\n",
        "import os\n",
        "import time\n",
        "from collections import namedtuple"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wyTT2vDTGlES",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.enable_v2_behavior()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrAW1XIdGmTc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from action_gap_rl import replay\n",
        "from action_gap_rl import value as value_lib\n",
        "from action_gap_rl import layers_lib\n",
        "replay = imp.reload(replay)\n",
        "value_lib = imp.reload(value_lib)\n",
        "layers_lib = imp.reload(layers_lib)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L2L_JOVpJm_l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class AttrDict(dict):\n",
        "  def __init__(self, *args, **kwargs):\n",
        "    super(AttrDict, self).__init__(*args, **kwargs)\n",
        "    self.__dict__ = self\n",
        "\n",
        "def to_dict(d):\n",
        "  if isinstance(d, AttrDict):\n",
        "    return {k: to_dict(v) for k, v in d.items()}\n",
        "  return d"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0C4zJX0fGo1e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def down_sample(lst, irresolution, reduce_fn):\n",
        "  return [reduce_fn(lst[i: i+irresolution]) for i in range(0, len(lst), irresolution)]\n",
        "\n",
        "first = lambda s: s[0]\n",
        "\n",
        "\n",
        "Metadata = namedtuple('Metadata', 'ds,y_scale,aux_data')\n",
        "Dataset = namedtuple('Dataset', 'o,a,r')\n",
        "\n",
        "def load_datasets(dataset_files, gamma=1.0, horizon=None, irresolution=1, scale=1.0):\n",
        "  datasets = {}\n",
        "  for df in DATASET_FILES:        \n",
        "    with open(df, 'rb') as f:\n",
        "      s = f.read()\n",
        "\n",
        "    memory = replay.Memory()\n",
        "    memory.unserialize(s)\n",
        "\n",
        "    if irresolution > 1:\n",
        "      memory.rewards = [down_sample(r, irresolution, sum) for r in memory.rewards]\n",
        "      memory.actions = [down_sample(a, irresolution, first) for a in memory.actions]\n",
        "      memory.observations = [down_sample(o, irresolution, first) for o in memory.observations]\n",
        "\n",
        "    if horizon is None:\n",
        "      qmax_Bax = np.reshape(value_lib.max_q_iteration(memory, gamma), (-1, 1))\n",
        "    else:\n",
        "      assert horizon == 1\n",
        "      qmax_Bax = np.reshape(memory.rewards, (-1, 1))\n",
        "    \n",
        "    data = Dataset(\n",
        "        o=memory.exited_states(), \n",
        "        a=memory.attempted_actions(),\n",
        "        r=qmax_Bax * scale)\n",
        "    \n",
        "    y_scale = np.max(np.abs(qmax_Bax * scale))\n",
        "\n",
        "    datasets[os.path.splitext(os.path.basename(df))[0]] = Metadata(ds=data, y_scale=y_scale, aux_data=memory.data)\n",
        "  \n",
        "  return datasets"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CqWE5rJ0GvBo",
        "colab_type": "code",
        "outputId": "7a09df52-8221-4e01-a6e7-ee491af782c7",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1582073959465,
          "user_tz": 480,
          "elapsed": 2158,
          "user": {
            "displayName": "Dan Abolafia",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAQmxyT8biPkQeDgz5lf2ocSzJsOqH8BqBuLz1a=s64",
            "userId": "05741471333872541970"
          }
        },
        "colab": {
          "height": 86
        }
      },
      "source": [
        "# Load datasets.\n",
        "DATASET_FILES = [\n",
        "    # TRAIN\n",
        "    '/tmp/action_gap_rl/datasets/v2/pendulum_train.pickle',\n",
        "\n",
        "    # EVAL\n",
        "    '/tmp/action_gap_rl/datasets/v2/pendulum_eval.pickle',\n",
        "]\n",
        "\n",
        "datasets = {\n",
        "    '1IR': load_datasets(DATASET_FILES, horizon=1, gamma=1.0, irresolution=1, scale=1/16),\n",
        "    '10IR': load_datasets(DATASET_FILES, horizon=1, gamma=1.0, irresolution=10, scale=1/109)\n",
        "}\n",
        "\n",
        "for category, subset in datasets.items():\n",
        "  for ds, data in subset.items():\n",
        "    print('{}/{} : {}, {}'.format(category, ds, data.ds.o.shape[0], data.y_scale))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1IR/pendulum_a2_t10_nnp_train : 22800, 1.0163658022882485\n",
            "1IR/pendulum_a2_t10_nnp_eval : 21800, 1.0168128733032964\n",
            "10IR/pendulum_a2_t10_nnp_train : 2280, 1.0021880235295113\n",
            "10IR/pendulum_a2_t10_nnp_eval : 2180, 1.0006529785472604\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DN1A5AJD8vZ3",
        "colab_type": "code",
        "outputId": "5e23664c-df02-4a33-9fed-e313c172d228",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1582073967090,
          "user_tz": 480,
          "elapsed": 37,
          "user": {
            "displayName": "Dan Abolafia",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAQmxyT8biPkQeDgz5lf2ocSzJsOqH8BqBuLz1a=s64",
            "userId": "05741471333872541970"
          }
        },
        "colab": {
          "height": 35
        }
      },
      "source": [
        "datasets['1IR']['pendulum_a2_t10_nnp_eval'].aux_data.keys()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['pi0_h=1/R0', 'pi0_h=1/R1', 'pi0_h=5/R0', 'pi0_h=5/R1', 'pi0_h=10/R0', 'pi0_h=10/R1'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DjtPee2wHFNX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class CategoricalPolicy(tf.keras.Model):\n",
        "  \"\"\"A policy that takes an arbitrary function as the un-normalized log pdf.\"\"\"\n",
        "\n",
        "  def __init__(self, config, name=None):\n",
        "    super(CategoricalPolicy, self).__init__(\n",
        "        name=name or self.__class__.__name__)\n",
        "    self._config = config\n",
        "    self.num_actions = config.num_actions\n",
        "    hidden_widths = config.hidden_widths\n",
        "    if config.embed:\n",
        "      transformation_layers = [layers_lib.soft_hot_layer(**config.embed)]\n",
        "    else:\n",
        "      transformation_layers = []\n",
        "    self._body = tf.keras.Sequential(\n",
        "        transformation_layers\n",
        "        + [tf.keras.layers.Dense(w, activation='relu') for w in hidden_widths]\n",
        "        + [tf.keras.layers.Dense(self.num_actions, activation=None)]\n",
        "    )\n",
        "\n",
        "  def call(self, states, actions):\n",
        "    # Returns unnormalized log-pdf of the actions (value predictions)\n",
        "    return index_rows(self._body(states), actions)\n",
        "\n",
        "  def argmax(self, states):\n",
        "    return tf.argmax(self._body(states), axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ydrNz8-Asc1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def l2_loss(model, states, actions, targets):\n",
        "  return tf.reduce_mean(tf.square(model(states, actions) - targets))\n",
        "\n",
        "def l1_loss(model, states, actions, targets):\n",
        "  return tf.reduce_mean(tf.abs(model(states, actions) - targets))\n",
        "\n",
        "\n",
        "def sample_batch(batch_size, *args):\n",
        "  assert args\n",
        "  idx = np.random.choice(args[0].shape[0], batch_size)\n",
        "  return tuple([arg[idx] for arg in args])\n",
        "\n",
        "\n",
        "def optimize(optimizer, model, loss_fn, data, eval_fn=None,\n",
        "             batch_size=100, maxiter=10000, report_gap=100, eval_size=None):\n",
        "  trace_loss = np.zeros(maxiter//report_gap + 1)\n",
        "  trace_eval = eval_fn and np.zeros(maxiter//report_gap + 1)\n",
        "  start = time.time()\n",
        "  j = 0\n",
        "  for i in range(maxiter+1):\n",
        "    optimizer.minimize(lambda: loss_fn(model, *sample_batch(batch_size, *data)),\n",
        "                       model.trainable_variables)\n",
        "    if i % report_gap == 0:\n",
        "      if eval_size:\n",
        "        batch = sample_batch(eval_size, *data)\n",
        "      else:\n",
        "        batch = data\n",
        "      trace_loss[j] = loss_fn(model, *batch).numpy()\n",
        "      if trace_eval is not None:\n",
        "        trace_eval[j] = eval_fn(model, *batch).numpy()\n",
        "        print(i, time.time() - start, trace_loss[j], trace_eval[j])\n",
        "      else:\n",
        "        print(i, time.time() - start, trace_loss[j])\n",
        "      j += 1\n",
        "  return trace_loss, trace_eval, time.time() - start\n",
        "\n",
        "\n",
        "def index_rows(a, idx):\n",
        "  # https://stackoverflow.com/a/40723732\n",
        "  idx_2 = tf.expand_dims(tf.cast(idx, tf.int32), 1)\n",
        "  rng = tf.expand_dims(tf.range(tf.shape(idx)[0]), 1)\n",
        "  ind = tf.concat([rng, idx_2], 1)\n",
        "  return tf.expand_dims(tf.gather_nd(a, ind), 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XiGsW8sYYzJL",
        "colab_type": "text"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BhRlRe8AAu7_",
        "colab_type": "code",
        "outputId": "a2706f86-057e-4a98-9508-08e78fdb27e7",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1581562610257,
          "user_tz": 480,
          "elapsed": 44171,
          "user": {
            "displayName": "Dan Abolafia",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAQmxyT8biPkQeDgz5lf2ocSzJsOqH8BqBuLz1a=s64",
            "userId": "05741471333872541970"
          }
        },
        "colab": {
          "height": 915
        }
      },
      "source": [
        "# device_string = '/device:GPU:0'\n",
        "# device_string = '/device:TPU:0'\n",
        "# device_string = ''  # CPU\n",
        "\n",
        "# '' for CPU, '/device:GPU:0' for GPU,  '/device:TPU:0' for TPU\n",
        "device_string = ''\n",
        "\n",
        "BATCH_SIZE = 1000\n",
        "ITERATIONS = 200  # 1000\n",
        "\n",
        "policies = {}\n",
        "train_datasets = ('1IR/pendulum_a2_t10_nnp_train', '10IR/pendulum_a2_t10_nnp_train')\n",
        "for catds in train_datasets:\n",
        "  cat, ds = catds.split('/')\n",
        "  data = datasets[cat][ds].ds\n",
        "  print('Training policy on \"{}\"'.format(catds))\n",
        "\n",
        "  embed=layers_lib.obs_embedding_kwargs(\n",
        "      20,\n",
        "      batch=data[0],\n",
        "  )\n",
        "  # embed=None\n",
        "  policy = CategoricalPolicy(AttrDict(num_actions=2, embed=embed, hidden_widths=[512,256]), name='policy_'+catds+'_'+str(np.random.randint(0, 1000000000)))\n",
        "\n",
        "  with tf.device(device_string):\n",
        "    print(l2_loss(policy, *sample_batch(1000, *data)).numpy())\n",
        "\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)\n",
        "  optimize(optimizer, policy, l2_loss, data, eval_fn=l1_loss,\n",
        "           batch_size=BATCH_SIZE,\n",
        "           maxiter=ITERATIONS,\n",
        "           report_gap=10)\n",
        "  policies[catds] = policy\n",
        "\n",
        "  print('\\n'+'='*32+'\\n')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training policy on \"1IR/pendulum_a2_t10_nnp_train\"\n",
            "0.1393086\n",
            "0 0.6871130466461182 0.10662606358528137 0.2695086896419525\n",
            "10 1.9298017024993896 0.024069122970104218 0.12149373441934586\n",
            "20 3.1672990322113037 0.007872077636420727 0.0693831667304039\n",
            "30 4.418404579162598 0.004413059912621975 0.04777473956346512\n",
            "40 5.662010669708252 0.002177769085392356 0.03612833842635155\n",
            "50 6.883200168609619 0.0012185540981590748 0.025759786367416382\n",
            "60 8.164255857467651 0.0008546442841179669 0.021938396617770195\n",
            "70 9.418537378311157 0.0006102664628997445 0.018048182129859924\n",
            "80 10.643690347671509 0.0004671918286476284 0.015309818089008331\n",
            "90 11.911546230316162 0.0003677069616969675 0.013742777518928051\n",
            "100 13.181400299072266 0.0003068363294005394 0.012310230173170567\n",
            "110 14.493411302566528 0.0002600151055958122 0.011396393179893494\n",
            "120 15.726702690124512 0.00022517552133649588 0.01056018564850092\n",
            "130 16.962790727615356 0.0001979361695703119 0.009940200485289097\n",
            "140 18.18386483192444 0.0001773530530044809 0.009301239624619484\n",
            "150 19.413865089416504 0.00015956826973706484 0.008901944383978844\n",
            "160 20.68570876121521 0.00014456317876465619 0.008485150523483753\n",
            "170 21.945855855941772 0.0001322800962952897 0.008065366186201572\n",
            "180 23.239185571670532 0.00012128549860790372 0.0077376943081617355\n",
            "190 24.49921178817749 0.00011158815323142335 0.007444859016686678\n",
            "200 25.755825757980347 0.00010345455666538328 0.007128487341105938\n",
            "\n",
            "================================\n",
            "\n",
            "Training policy on \"10IR/pendulum_a2_t10_nnp_train\"\n",
            "0.34569624\n",
            "0 0.2002716064453125 0.2904421091079712 0.4757387340068817\n",
            "10 1.056138038635254 0.045797199010849 0.16910883784294128\n",
            "20 1.8789281845092773 0.03701580688357353 0.14168091118335724\n",
            "30 2.6946518421173096 0.014611796475946903 0.08349346369504929\n",
            "40 3.789968252182007 0.012578931637108326 0.07779631018638611\n",
            "50 4.639315843582153 0.00848556961864233 0.05789828300476074\n",
            "60 5.495704889297485 0.006276874803006649 0.04842951521277428\n",
            "70 6.334512233734131 0.004708305932581425 0.041257236152887344\n",
            "80 7.178510904312134 0.003511307528242469 0.035275086760520935\n",
            "90 8.02871298789978 0.0026497822254896164 0.030110469087958336\n",
            "100 8.873443365097046 0.0020326655358076096 0.026405006647109985\n",
            "110 9.70754337310791 0.0015879484126344323 0.023577705025672913\n",
            "120 10.541680812835693 0.001278302283026278 0.020893214270472527\n",
            "130 11.37149453163147 0.0010631912155076861 0.018960315734148026\n",
            "140 12.201504945755005 0.0008879590313881636 0.017184384167194366\n",
            "150 13.037904024124146 0.0007555758347734809 0.015799270942807198\n",
            "160 13.878431558609009 0.0006492193206213415 0.014735614880919456\n",
            "170 14.714341163635254 0.000563266163226217 0.013822839595377445\n",
            "180 15.557454586029053 0.00049587432295084 0.012855799868702888\n",
            "190 16.405006408691406 0.0004416673327796161 0.012297489680349827\n",
            "200 17.247861623764038 0.0003948996018152684 0.011713462881743908\n",
            "\n",
            "================================\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xTVcAz8LY0iQ",
        "colab_type": "text"
      },
      "source": [
        "# Evaluation on dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "htQ65O84dZRk",
        "colab_type": "code",
        "outputId": "bba02876-35ee-46ad-f57e-1c52acd50541",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1581630089550,
          "user_tz": 480,
          "elapsed": 46,
          "user": {
            "displayName": "Dan Abolafia",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAQmxyT8biPkQeDgz5lf2ocSzJsOqH8BqBuLz1a=s64",
            "userId": "05741471333872541970"
          }
        },
        "colab": {
          "height": 52
        }
      },
      "source": [
        "policies"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'1IR/pendulum_a2_t10_nnp_train': <__main__.CategoricalPolicy at 0x7f093e86eba8>,\n",
              " '10IR/pendulum_a2_t10_nnp_train': <__main__.CategoricalPolicy at 0x7f093e61bc88>}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fkiVu96TA1PE",
        "colab_type": "code",
        "outputId": "e1faa59d-ebc7-46c6-a412-b48da0acf519",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1581562613731,
          "user_tz": 480,
          "elapsed": 678,
          "user": {
            "displayName": "Dan Abolafia",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAQmxyT8biPkQeDgz5lf2ocSzJsOqH8BqBuLz1a=s64",
            "userId": "05741471333872541970"
          }
        },
        "colab": {
          "height": 155
        }
      },
      "source": [
        "for catds, policy in policies.items():\n",
        "  cat, ds = catds.split('/')\n",
        "  assert ds.endswith('_train')\n",
        "  ds = ds[:-6]+'_eval'\n",
        "  print('evaluating policy for {}/{}'.format(cat, ds))\n",
        "  eval_data = datasets[cat][ds].ds\n",
        "  print('L1: ', l1_loss(policy, *eval_data).numpy())\n",
        "  print('L2: ', l2_loss(policy, *eval_data).numpy())\n",
        "  print('')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "evaluating policy for 1IR/pendulum_a2_t10_nnp_eval\n",
            "L1:  0.0072429325\n",
            "L2:  0.0001110887\n",
            "\n",
            "evaluating policy for 10IR/pendulum_a2_t10_nnp_eval\n",
            "L1:  0.014524726\n",
            "L2:  0.00071514945\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8hUadUVpY7dk",
        "colab_type": "code",
        "outputId": "eff02802-cfc4-4108-81f3-eb2d1021e803",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1581562614060,
          "user_tz": 480,
          "elapsed": 319,
          "user": {
            "displayName": "Dan Abolafia",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAQmxyT8biPkQeDgz5lf2ocSzJsOqH8BqBuLz1a=s64",
            "userId": "05741471333872541970"
          }
        },
        "colab": {
          "height": 363
        }
      },
      "source": [
        "# Sanity check: view prediction errors on eval sets \n",
        "for catds, policy in policies.items():\n",
        "  cat, ds = catds.split('/')\n",
        "  assert ds.endswith('_train')\n",
        "  ds = ds[:-6]+'_eval'\n",
        "  eval_data = datasets[cat][ds].ds\n",
        "  states, actions, targets = eval_data\n",
        "  print(ds)\n",
        "  print(policy(states, actions) - targets)\n",
        "  print('')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "pendulum_a2_t10_nnp_eval\n",
            "tf.Tensor(\n",
            "[[-0.00222814]\n",
            " [-0.0005995 ]\n",
            " [ 0.00530088]\n",
            " ...\n",
            " [ 0.00438103]\n",
            " [ 0.00365618]\n",
            " [ 0.00426745]], shape=(19600, 1), dtype=float32)\n",
            "\n",
            "pendulum_a2_t10_nnp_eval\n",
            "tf.Tensor(\n",
            "[[-0.00349784]\n",
            " [ 0.00062013]\n",
            " [ 0.01205844]\n",
            " ...\n",
            " [-0.00655425]\n",
            " [ 0.0416511 ]\n",
            " [ 0.0086602 ]], shape=(1960, 1), dtype=float32)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3RDxX-E_eJMy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}